# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import time, json, pymysql, re
import hashlib
import uuid
import gzip

from twisted.enterprise import adbapi


class MysqlPipeline(object):
    """docstring for MysqlTwistedPipeline"""

    def __init__(self, client):
        self.client = client
        self.cursor = self.client.cursor()

    @classmethod
    def from_settings(cls, settings):
        # 会被scrapy调用的类方法,返回一个dbpoll对象传递给MysqlTwistedPipeline类,构建新实例
        # dbpool = adbapi.ConnectionPool("pymysql",
        #                                host=settings["MYSQL_HOST"],
        #                                db=settings["MYSQL_DBNAME"],
        #                                user=settings["MYSQL_USER"],
        #                                password=settings["MYSQL_PASSWORD"],
        #                                charset="utf8",
        #                                cursorclass=pymysql.cursors.DictCursor,
        #                                use_unicode=True)
        client = pymysql.connect(host=settings["MYSQL_HOST"],		# 服务器
                                 port=settings["MYSQL_PORT"],		# 端口
                                 user=settings["MYSQL_USER"],  		# 用户名
                                 passwd=settings["MYSQL_PASSWORD"],	# 密码
                                 db=settings["MYSQL_DBNAME"],  		# 数据库名
                                 charset='utf8')

        return cls(client)

    def process_item(self, item, spider):
        # 处理item 通过runInteraction执行异步操作,第一个参数是具体的操作过程,第二个参数是操作数据
        # 并返回一个query,query是Deferred类的实例,调用其中的addErrback方法处理异常
        guid = str(uuid.uuid1())
        sha1 = str_encrypt(item['html'])
        html = item['html']
        title = item['title']
        node_id = item['node_id']
        site_id = item['site_id']

        url = item['url']
        insert_sql = 'insert into page(Id,NodeId,SiteId,Url,SHA1,CreationTime,Length,Title,Html) values(%s,%s,%s,%s,%s,%s,%s,%s,%s)'
        self.cursor.execute(insert_sql, (guid, node_id, site_id, url, sha1, time.strftime("%Y-%m-%d %H:%M:%S"), len(html), title, html))
        self.client.commit()
		
        return item

def str_encrypt(text):
    """
    使用sha1加密算法，返回str加密后的字符串
    """
    sha = hashlib.sha1(text)
    encrypts = sha.hexdigest()
    return encrypts
